#!/usr/bin/env python
# coding=utf-8
'''
福建省中国旅行网
国内旅游：http://www.ctsfj.com/line/list.asp?id=1  
出境线路：http://www.ctsfj.com/line/list.asp?id=2
港澳台：http://www.ctsfj.com/line/list.asp?id=11
金马澎：http://www.ctsfj.com/line/list.asp?id=22
港澳散拼：http://www.ctsfj.com/line/list.asp?id=26
福州周边：http://www.ctsfj.com/line/list.asp?id=12  
自由专线（福州境内） http://www.ctsfj.com/line/list.asp?id=25
'''
import os, sys, re, traceback, logging
import urllib
from datetime import *
import time
import datetime
import getopt
import urllib2
from Route import *

vacation_routenum = re.compile(r'''<span\s*class=.mypage_font.>\s*([0-9]+)\s*</span>''', re.X|re.S)

vacation_routeurl = re.compile(r'''<td\s*width=.428.>\s*<a\s*href="([\S\s]+?)"\s*[\S\s]+?>([\S\s]+?)</a>''', re.X|re.S)

route_price_re = re.compile(r'''<td\s*height="25"\s*colspan="2"\s*class="font3">[\S\s]+?：￥([\S\s]+?[0-9]+)''', re.X|re.S)
route_sights_re = re.compile(r'''<td\s*width="50%"\s*class="font3">([\S\s]+?)</td>''', re.X|re.S)
route_depcity_re = re.compile(r'''<td\s*width="50%"\s*height="25"\s*class="font3">([\S\s]+?)</td>''', re.X|re.S)

route_days_re = re.compile(r'''</tr>\s*<tr>\s*<td\s*height="25"\s*colspan="2"\s*class="font3">[\S\s]+?([0-9]+)''', re.X|re.S)
route_detail_re = re.compile(r'''class='14_7AB900_1'[\S\s]+?>([\S\s]+?)</a>''', re.X|re.S)

route_dates_re = re.compile(r'''出发日期：([\S\s]+?)</td>''', re.X|re.S)

route_planes_re = re.compile(r'''<div\s*class=.t_detail_trip_text[^>]+>\s*([\S\s]+?)\s*</div>''', re.X|re.S)

route_trip_all_re = re.compile(r'''<div\s*class=.t_detail_trip_all2[^>]+>\s*([\S\s]+?)\s*</div>''', re.X|re.S)

re_h=re.compile('<[\S\s]*>') #取出信息中夹杂的html标签

http_url = "http://www.ctsfj.com/line/"  #福建中国旅行社
route_telphone = "968877"

def do_crawl():
    try:
        server_url_data = [
                ('国内旅游','跟团游','http://www.ctsfj.com/line/list.asp?id=1&page=1', ),
                ('出境线路','跟团游','http://www.ctsfj.com/line/list.asp?id=2&page=1', ),
                ('港澳台','跟团游','http://www.ctsfj.com/line/list.asp?id=11&page=1', ),
                ('港澳散拼','跟团游','http://www.ctsfj.com/line/list.asp?id=26&page=1', ),
                ('金马澎','跟团游','http://www.ctsfj.com/line/list.asp?id=22&page=1', ),
                ('福州周边','跟团游','http://www.ctsfj.com/line/list.asp?id=12&page=1', ),
                ('福州周边（自由专线）','自由行','http://www.ctsfj.com/line/list.asp?id=25&page=1', ),
        ]
        #
        #
        for route_te,routetype,server_url in server_url_data:
            routenum=''
            html = urllib2.urlopen(server_url).read()
            vacation_route_num = vacation_routenum.search(html)
            if vacation_route_num:
                u=util.strutil(vacation_route_num.group(1))
                routenum=u.filter_tags()
                routenum = int(routenum)/8+1
                for i in range(routenum):
                    page_url = server_url[:-1]+str(i+1)
                    print "page_url:"+page_url    
                    
                    '''获取route_url'''
                    vacation_route_url=""
                    html = urllib2.urlopen(page_url).read()
                    html = getUTF(html)
                    vacation_route_url = vacation_routeurl.findall(html)
                    for i,route_title in vacation_route_url:
                        try:
                            print "route_url:"+http_url+i
                            print "route_title:"+route_title
                            route_url = http_url+i
                            #route_url = "http://www.ctsfj.com/line/show.asp?id=552"
                            #此处开始获取线路信息
                            html = urllib2.urlopen(route_url).read()
                            html = getUTF(html)
                            #
                            route_sights = route_sights_re.findall(html)
                            if route_sights:
                                if len(route_sights[0].split("："))>1:
                                    #print route_sights[0].split("：")[1]
                                    route_sights = route_sights[0].split("：")[1]
                            #
                            depcity = route_depcity_re.findall(html)
                            if depcity:
                                #print depcity[0].split("：")[1]
                                depcity = depcity[0].split("：")[1]
                            #
                            route_price = route_price_re.findall(html)
                            if route_price:
                                route_price = route_price[0].replace("特价","").replace("心动价","")
                                print route_price
                            #
                            route_days = route_days_re.findall(html)
                            if route_days:
                                route_days = route_days[1]
                                #print route_days
                            #
                            route_detail = route_detail_re.findall(html)
                            if route_detail:
                                route_detail = route_detail[0]
                                #print route_detail
                            #
                            route_dates=""
                            date1 = date.today()
                            route_date = route_dates_re.findall(html)
                            if route_date:
                                print route_date[0]
                                route_date = route_date[0]
                                if route_date.find("天天")>-1:
                                    route_dates = getdates()
                                if route_date.find("月")>-1:
                                    route_dates = route_date.replace("日","").replace("发","").replace("团","").split("月")
                                    nowDate = time.strptime(str(date1), "%Y-%m-%d")
                                    year = nowDate[0]
                                    mon = route_dates[0]
                                    day = route_dates[1]
                                    if day.find("、")>-1:
                                        day = day.split("、")[1]
                                    print mon
                                    print day
                                    d1 = datetime.date(int(year), int(mon), int(day))
                                    if str(d1)<str(date1):
                                        continue
                                    route_dates = str(d1)+";"
                                if route_date.find("20")>-1 and len(route_date)==8:
                                    route_dates = route_date[:4]+"-"+route_date[4:6]+"-"+route_date[6:]+";"
                                    if route_dates.replace(";","")<str(date1):
                                        continue
                            print route_dates+"=========="
                            #
                            route_plane = getPlane(html)
                            #print route_plane
                            #
                            route_hotel = ""
                            route_meal = ""
                            route_traffic01 = ""
                            route_tickets = ""
                            route_payown = ""
                            route_trip = route_trip_all_re.findall(html)
                            if route_trip:
                                for route_trips in route_trip[0].split("<br />"):
                                    if route_trips.find("住宿")>-1:
                                        if route_trips.find("：")>-1:
                                            route_trips = (route_trips.split("："))[1]
                                        route_hotel = re_h.sub('',route_trips)
                                    if route_trips.find("餐饮")>-1:
                                        if route_trips.find("：")>-1:
                                            route_trips = (route_trips.split("："))[1]
                                        route_meal = re_h.sub('',route_trips)
                                    if route_trips.find("交通")>-1:
                                        if route_trips.find("：")>-1:
                                            route_trips = (route_trips.split("："))[1]
                                        route_traffic01 = re_h.sub('',route_trips)
                                    if route_trips.find("门票")>-1:
                                        if route_trips.find("：")>-1:
                                            route_trips = (route_trips.split("："))[1]
                                        route_tickets = re_h.sub('',route_trips)
                                route_payown = route_trip[1].strip().replace("<p>","").replace("</p>","")
                                #print route_payown
                            #
                            route_go_back = getTrafficType(route_title)
                            #print route_go_back
                            #入库
                            route = Route()
                            route.site = '34'
                            route.supplier_url = "http://www.ctsfj.com/"
                            route.supplier = "福建省中国旅行社"
                            route.title = route_title
                            route.rout_url = route_url
                            route.outcity = depcity
                            route.sights = route_sights
                            route.telphone = route_telphone
                            route.detail = route_detail
                            route.dest = route_sights
                            route.guide = ''
                            route.type = routetype
                            route.dates = route_dates
                            route.price = int(route_price)
                            route.go_t = route.back_t = route_go_back
                            route.plane = route_plane
                            route.days = route_days

                            route.hotel = route_hotel
                            route.meal = route_meal
                            route.traffic01 = route_traffic01
                            route.tickets = route_tickets
                            route.payown = route_payown
                            
                            getschedules = getSchedules(html)
                            if getschedules=='no':
                                print "==no schedule=="
                            else:
                                route.schedules = getschedules
                                route.updRoute_Schedules()
                            route.updRoute()
                            
                            print "++++++"
                            #print "++++++"
                            #print "++++++"
                        except Exception,e:
                            print traceback.format_exc(sys.exc_info())
                        #print routenum+"dddd"
                
    except:
        print traceback.format_exc(sys.exc_info())


Schedules_place_re = re.compile(r'''<td\s*height="31"\s*align="left"\s*valign="top"\s*class="show">([\S\s]+?)</td>''', re.X|re.S)
Schedules_brief_re = re.compile(r'''<div\s*class=.t_detail_trip_text[^>]+>\s*([\S\s]+?)\s*</div>\s*<div''', re.X|re.S)
Schedules_eat_tra_hot_re = re.compile(r'''<div\s*class=.t_detail_trip_text2[^>]+>\s*([\S\s]+?)\s*</div>''', re.X|re.S)

def getSchedules(html):
     
     schedules_place_list = Schedules_place_re.findall(html)
     if schedules_place_list:
         schedules_place_list = strip_tags(schedules_place_list[1])
         #print schedules_place_list
         if schedules_place_list.find('第')>-1:
             schedules = range(0,len(schedules_place_list))
             for i in range(1,20):
                 schedule = Schedule()
                 place_lists=""
                 place_lists = schedules_place_list.split("第"+str(i)+"天")
                 if len(place_lists)>1:
                     #print place_lists[1]
                     schedule.s_brief = place_lists[1].strip()
                     schedule.s_num = i
                 else:
                     return "no"
                 schedules[i-1]=schedule
             return schedules
         return "no"
     return "no"    
def getPlane(html):
     plane_str = ''
     #判断是否含有线路行程天数信息
     place_list = Schedules_place_re.findall(html)
     if place_list:
         place_list = strip_tags(place_list[1])
         #print place_list
         if place_list.find('第')>-1:
             for i in range(1,20):
                 place_lists=""
                 place_lists = place_list.split("第"+str(i)+"天")
                 if len(place_lists)>1:
                     #print place_lists[1]
                     plane_str = plane_str + "#第" + str(i) + "天# "
                     plane_str = plane_str + place_lists[1].strip()
     return plane_str

def getTrafficType(t):
    t1='-'
    if t.find('汽')>-1:
        t1='汽车'
    if t.find('动')>-1:
        t1='动车'
    if t.find('卧')>-1:
        t1='卧铺'
    if t.find('火车')>-1:
        t1='火车'
    if t.find('船')>-1:
        t1='邮轮'
    if t.find('邮')>-1:
        t1='邮轮'
    if t.find('飞')>-1:
        t1='飞机'
    if t.find('机')>-1:
        t1='飞机'
    return t1

#去掉文本标签
def strip_tags(html):
    from HTMLParser import HTMLParser
    html=html.strip()
    html=html.strip("\n")
    result=[]
    parse=HTMLParser()
    parse.handle_data=result.append
    parse.feed(html)
    parse.close()
    return "".join(result)

def getUTF(gb):
    #uni=unicode(gb,'gb2312')
    utf = gb.decode('gb2312','ignore').encode('utf-8')
    return utf

def getdates():
    #天天发团是 使用该方法
    #from datetime import *  #将此行放在import datetime上边
    #days=i+1  days=?  就是距当前日期的天数。
    route_dates = ""
    date1 = date.today()
    for i in range(10):
        d=datetime.timedelta(days=i+1)
        route_dates = str(route_dates)+str(date1+d)+ ";"
    #print route_dates
    return route_dates

if __name__ == '__main__':
    script_path = os.path.dirname(os.path.realpath(__file__))
    os.chdir(script_path)
    do_debug = False

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], 'hdno:p:', ['help', 'debug', 'no-cache', 'only-cache', 'provider'])
    except getopt.GetoptError:
        usage()
        sys.exit()

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage()
            sys.exit()
        elif opt in ('-d', '--debug'):
            do_debug = True
        elif opt in ('-n', '--no-cache'):
            use_cache = False
        elif opt in ('-o', '--only-cache'):
            if arg.lower() in ('no', 'n', '0'):
                only_cache = False
            else:
                only_cache = True
        elif opt in ('-p', '--provider'):
            pass

    if do_debug:
        import pdb
        pdb.set_trace()
    do_crawl()